package com.chance.cc.crawler.development.bootstrap.du;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.*;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.scripts.du.DUProductCrawlerScript;
import com.chance.cc.crawler.development.scripts.du.DUProductNewCrawlerScript;
import com.chance.cc.crawler.development.utils.RedisReader;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileInputStream;
import java.io.IOException;
import java.util.*;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.turnPage;
import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.Tag_Field_Domain_Result_Json;
import static org.codehaus.groovy.runtime.EncodingGroovyMethods.md5;

/**
 * @author lt
 * @version 1.0
 * @date 2021-05-31 13:24:21
 * @email okprog@sina.com
 */
public class DuProductTest {
    public static Logger logger = LoggerFactory.getLogger(DUSellCalenderTest.class);

    public static final String scKey = "d245a0ba8d678a61";
    public static final String categoryDetail = "https://app.dewu.com/api/v1/app/commodity/ice/search/doCategoryDetail"; //分类url
    public static final String productDetail = "https://app.dewu.com/api/v1/app/index/ice/flow/product/detail";
    public static final String productDetailUrl = "https://app.dewu.com/api/v1/h5/index/fire/flow/product/detail";

    public static final String metaSite = "product";
    public static final String startUrl = "api/v1/h5/commodity/fire/search/doCategoryDetail";
    public static final String domain = "du_app";

    public static final String site = "product";
    public static final String site_biz = "product";
    public static final String crawler_type = "crawler_type";
    public static final String tagBrandName = "brand_name";
    public static final String tagBrandId = "brand_id";
    public static final String tagRecommendId = "recommend_id";
    public static final String tagSpuIds = "spu_ids";
    public static final String tagProductId = "product_id";
    public static final String tagProductDetail = "product_detail";
    public static final String tagSellProduct = "sell_product";
    public static final String tagAskProduct = "ask_product";

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }
    public static void main(String[] args) throws IOException {
//        testSign();
//        doCategoryDetail();
//        doProductDetail();
//        testDUSpuIdInit();
//        testWXProductDetail();
//        System.out.println(RandomUtils.nextInt(1000,9999));
//        System.out.println("{\"Product Code\": \"TJG138-9090\", \"productId\": \"DU62953\", \"Brand\": \"Asics\", \"Category\": \"潮鞋\", \"category_two\": \"Asics\"}".getBytes().length);
//        findDelId();
        importProductDeatil2Record();
    }

    public static void findDelId(){
        String delResultQueue = "delResult-du_app-spuIds";
        String delResultQueue2 = "delResult-du_app-spuIds2";
        String findDelId = "1006920";
        RedisReader redisReader = new RedisReader("192.168.1.214",6379,3);
        while (true){
            String str = redisReader.listPop(delResultQueue);
            JSONObject jsonObject = JSONObject.parseObject(str);
            String recordKey = jsonObject.getString("recordKey");
            if (recordKey.endsWith(findDelId)){
                System.out.println(jsonObject);
                redisReader.listPush(delResultQueue2,str);
                break;
            }
            redisReader.listPush(delResultQueue2,str);
        }
    }

    public static void testWXProductDetail(){
        HttpConfig httpConfig = HttpConfig.me("test");
        httpConfig.setProxy(proxy);
        httpConfig.setUseProxy(true);

        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());

        for (int i = 0; i < 1; i++) {
            new Thread(new TestWXProductDetail(downloader,httpConfig,"1584214"),"thread_" + i).start();
        }
    }

    public static void testSign(){
        String times = String.valueOf(System.currentTimeMillis());
        logger.info("times is [{}]",times);
        Map<String,Object> map = new HashMap<>();
        map.put("tabId","020300");
        map.put("lastId","");
        map.put("limit","20");
        map.put("abVideoCover","1000");
        map.put("abRectagFengge","0");
        map.put("newSign","");
        Map<String, Object> objectMap = genSignNeedParams(map, times);
        String newSignature = genNewSignature(objectMap, scKey);
        logger.info("newSign is : [{}]",newSignature);

    }

    private static Map<String, Object> genSignNeedParams(Map<String, Object> urlParams,String timestamp) {
        Map<String, Object> params = new HashMap<>();
        Set<Map.Entry<String, Object>> entries = urlParams.entrySet();
        for (Map.Entry<String, Object> entry : entries) {
            params.put(entry.getKey(),entry.getValue());
        }

        params.put("uuid","fac3142579391d98");
        params.put("platform","android");
        params.put("v","4.71.0");
        params.put("loginToken","");
        params.put("timestamp",timestamp);//header中的时间戳
        return params;
    }


    public static void doCategoryDetail(){
        HttpConfig httpConfig = HttpConfig.me("test");
        httpConfig.setProxy(proxy);
        httpConfig.setUseProxy(true);

        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());

        List<String> keywords = Arrays.asList("3", "6");

        for (String keyword : keywords) {
            new Thread(new TestDoCategoryDetail(downloader, httpConfig, keyword), "thread_keyword-" + keyword).start();
        }
    }


    public static void doProductDetail(){
        HttpConfig httpConfig = HttpConfig.me("test");
//        httpConfig.setProxy(proxy);
//        httpConfig.setUseProxy(true);

        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());

        for (int i = 0; i < 1; i++) {
            new Thread(new TestProductDetail(downloader,httpConfig,"1526986"),"thread_" + i).start();
        }
    }

    public static void testDUSpuIdInit(){
        HttpConfig httpConfig = HttpConfig.me("test");
        httpConfig.setProxy(proxy);
        httpConfig.setUseProxy(true);

        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());

        for (int i = 0; i < 1; i++) {
            new Thread(new TestDUSpuIdInit(downloader,httpConfig),"thread_" + i).start();
        }
    }

    private static void importProductDeatilRecord() throws IOException {

        RedisReader redisReader = new RedisReader("192.168.1.215",6379,4);

//        List<String> lines = IOUtils.readLines(new FileInputStream("C:\\Users\\Administrator\\Desktop\\du_app_product_detail.json"), "utf-8");
        List<String> lines = IOUtils.readLines(new FileInputStream("C:\\Users\\Administrator\\Desktop\\du_app_product_detail_tmp.json"), "utf-8");
        for (String line : lines) {
            if (line.startsWith("####")){
                continue;
            }
            CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                    .startPageRequest(domain, turnPage)
                    .httpUrl(startUrl)
                    .releaseTime(System.currentTimeMillis())
                    .needWashed(false)
                    .needParsed(true)
                    .filter(CrawlerEnum.CrawlerRecordFilter.key)
                    .addFilterInfo(FilterUtils.redisFilterKeyInfo("filter-du_app-spuIds"))
                    .proxy(proxy)
                    .build();

            requestRecord.getHttpConfig().setProxyQueueSize(100);
            requestRecord.getHttpConfig().setProxyType("queue");

            requestRecord.tagsCreator().bizTags().addSite(metaSite);
            requestRecord.tagsCreator().bizTags().addSiteBiz(metaSite);
            requestRecord.tagsCreator().bizTags().addCustomKV("crawler_type","productDetail");

            CrawlerRequestRecord initRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(requestRecord)
                    .httpUrl("https://app.dewu.com/api/v1/h5/index/fire/flow/product/detail")
                    .recordKey("https://app.dewu.com/api/v1/h5/index/fire/flow/product/detail")
                    .releaseTime(System.currentTimeMillis())
                    .needWashed(false)
                    .needParsed(true)
                    .copyBizTags()
                    .build();

            genInitProductDetailRecord(initRecord,initRecord.getHttpRequest(),JSONObject.parseObject(line));
            String shaKey = DigestUtils.sha1Hex(System.currentTimeMillis()+initRecord.getRecordKey());

            redisReader.mapPush("crawler-du_app-product-new-productDetail-queue_map",shaKey,JSON.toJSONString(initRecord));
            redisReader.listPush("crawler-du_app-product-new-productDetail-queue_list",shaKey);
        }


    }


    private static void importProductDeatil2Record() throws IOException {

        RedisReader redisReader = new RedisReader("192.168.1.215",6379,4);

//        List<String> lines = IOUtils.readLines(new FileInputStream("C:\\Users\\Administrator\\Desktop\\du_app_product_detail.json"), "utf-8");
        List<String> lines = IOUtils.readLines(new FileInputStream("C:\\Users\\Administrator\\Desktop\\du_app_product_detail_tmp.json"), "utf-8");
        for (String line : lines) {
            if (line.startsWith("####")){
                continue;
            }
            CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                    .startPageRequest(domain, turnPage)
                    .httpUrl(startUrl)
                    .releaseTime(System.currentTimeMillis())
                    .needWashed(true)
                    .needParsed(true)
                    .proxy(proxy)
                    .build();

            requestRecord.getHttpConfig().setProxyQueueSize(100);
            requestRecord.getHttpConfig().setProxyType("queue");

            requestRecord.tagsCreator().bizTags().addDomain(domain);
            requestRecord.tagsCreator().bizTags().addSite(DUProductNewCrawlerScript.site);
            requestRecord.tagsCreator().bizTags().addSiteBiz(DUProductNewCrawlerScript.site_biz);
            requestRecord.tagsCreator().bizTags().addCustomKV("crawler_type","productDetail");

            CrawlerRequestRecord initRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(requestRecord)
                    .httpUrl("https://app.dewu.com/api/v1/h5/index/fire/flow/product/detail")
                    .recordKey("https://app.dewu.com/api/v1/h5/index/fire/flow/product/detail")
                    .releaseTime(System.currentTimeMillis())
                    .needWashed(true)
                    .needParsed(true)
                    .copyBizTags()
                    .build();

            genInitProductDetailRecord(initRecord,initRecord.getHttpRequest(),JSONObject.parseObject(line));
            String shaKey = DigestUtils.sha1Hex(System.currentTimeMillis()+initRecord.getRecordKey());

            redisReader.mapPush("crawler-du_app-product_detail_new-queue_map",shaKey,JSON.toJSONString(initRecord));
            redisReader.listPush("crawler-du_app-product_detail_new-queue_list",shaKey);
        }


    }

    private static void genInitProductDetailRecord(CrawlerRequestRecord requestRecord, HttpRequest httpRequest, JSONObject contentObj) {
        String spuId = contentObj.getString("spuId");
        String cateId = contentObj.getString("cateId");
        String category = contentObj.getString("category");
        String brandId = contentObj.getString("brandId");
        String brandName = contentObj.getString("brandName");
        httpRequest.setMethod(HttpConstant.Method.POST);
        requestRecord.setRecordKey(StringUtils.joinWith("-",domain,metaSite, CrawlerEnum.CrawlerDataType.article.enumVal(),spuId));
        requestRecord.setReleaseTime(System.currentTimeMillis());
        Map<String,Object> params = new HashMap<>();
        params.put("spuId",spuId);
        params.put("productSourceName","");
        params.put("propertyValueId","0");

        String sign = DUProductCrawlerScript.genSignature(params);
        params.put("sign",sign);
        HttpRequestBody httpRequestBody = HttpRequestBody.json(JSON.toJSONString(params), "utf-8");
        httpRequest.setRequestBody(httpRequestBody);

        httpRequest.addHeader("Host","app.dewu.com");
        httpRequest.addHeader("Connection","keep-alive");
        httpRequest.addHeader("User-Agent",genRandomUA());
        httpRequest.addHeader("Content-Type","application/json");
        httpRequest.addHeader("platform","h5");
        httpRequest.addHeader("AppId","wxapp");
        httpRequest.addHeader("appVersion","4.4.0");
        httpRequest.addHeader("charset","utf-8");
        httpRequest.addHeader("Accept-Encoding","gzip,compress,br,deflate");
        httpRequest.addHeader("Referer","https://servicewechat.com/wx3c12cdd0ae8b1a7b/225/page-frame.html");
        httpRequest.addHeader("wxapp-login-token","");
        requestRecord.tagsCreator().bizTags().getCategoryTag().getKvTags().remove(Tag_Field_Domain_Result_Json); //移除
        requestRecord.tagsCreator().requestTags().removeRequestType(CrawlerEnum.CrawlerRequestType.filter);
        requestRecord.setRecordKey(StringUtils.joinWith("-",domain,site,spuId));
        requestRecord.tagsCreator().bizTags().addCustomKV(crawler_type,"productDetail");
        requestRecord.tagsCreator().bizTags().addCustomKV(tagProductId,spuId);
        requestRecord.tagsCreator().bizTags().addCustomKV("cateId",cateId);
        requestRecord.tagsCreator().bizTags().addCustomKV("category",category);
        requestRecord.tagsCreator().bizTags().addCustomKV(tagBrandId,brandId);
        requestRecord.tagsCreator().bizTags().addCustomKV(tagBrandName,brandName);
    }

    private static String genRandomUA(){
        int part1 = RandomUtils.nextInt(1000, 9999);
        int part2 = RandomUtils.nextInt(1000, 1999);
        int part3 = RandomUtils.nextInt(100, 999);

        return String.format("Mozilla/5.0 (Linux; Android 8.0.0; MI 5s Build/OPR1.170623.032; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.62 XWEB/2797 MMWEBSDK/20210501 Mobile Safari/537.36 MMWEBID/%s MicroMessenger/8.0.6.%s(0x28000%s) Process/appbrand0 WeChat/arm64 Weixin NetType/WIFI Language/zh_CN ABI/arm64 MiniProgramEnv/android",
                part1,part2,part3);
    }

    static class TestDUSpuIdInit implements Runnable{

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        Set<String> spuIds = new HashSet<>();
        List<String> dupSpuIds = new ArrayList<>();
        List<String> failedSpuIds = new ArrayList<>();
        RedisReader redisReader = new RedisReader("192.168.1.214",6379,3);
//        RedisReader redisReader2 = new RedisReader("192.168.1.214",6379,3);
        public static final String spuIdsQueue = "filter-du_app-spuIds";
        public static final String duResultList = "result-du_app-spuIds";


        public TestDUSpuIdInit(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {
//            while (true){
//                try {
//                    String result = redisReader.listPop(duResultList);
//                    redisReader2.listPush(duResultList,result);
//                } catch (Exception e) {
//                    break;
//                }
//            }

            try {
                List<String> allSpuIds = IOUtils.readLines(new FileInputStream("D:\\chance\\log\\du\\du_product_add.json"),"utf-8");
                logger.info("原始SpuIds大小为： [{}]",allSpuIds.size());
                for (String allSpuId : allSpuIds) {
                    JSONObject jsonObject = JSONObject.parseObject(allSpuId);
                    String dataId = jsonObject.getString("dataId");
                    boolean isContains = redisReader.sismember(spuIdsQueue, dataId);
                    if (isContains){
                        dupSpuIds.add(dataId);
                        logger.info("duplicate dataId : [{}]",dataId);
                        continue;
                    }
                    redisReader.sadd("filter-du_app-spuIds",dataId);
                    Long listLen = redisReader.listPush(duResultList, allSpuId);
                    logger.info("累计商品ID总数：[{}]", listLen);
                }
                logger.info("原始SpuIds大小为： [{}]",allSpuIds.size());
                logger.info("存在重复值：[{}]",dupSpuIds.size());
            } catch (IOException e) {
                logger.error(e.getMessage());
            }
        }
    }


    static class TestDoCategoryDetail implements Runnable {

        HttpClientDownloader downloader;
        HttpConfig httpConfig;
        String cateId;

        Set<String> spuIds = new HashSet<>();

        public TestDoCategoryDetail(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        public TestDoCategoryDetail(HttpClientDownloader downloader, HttpConfig httpConfig,String cateId) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
            this.cateId = cateId;
        }

        @Override
        public void run() {
            String times = String.valueOf(System.currentTimeMillis());
            logger.info("times is [{}]",times);
            Map<String,Object> map = new HashMap<>();
            map.put("catId",cateId);
            map.put("loginToken","");
            map.put("newSign","");
            map.put("platform","android");
            map.put("timestamp",times);
            map.put("uuid","fac3142579391d98");
            map.put("v","4.69.5");
            String newSignature = genNewSignature(map, scKey);
            logger.info("newSign is : [{}]",newSignature);

            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl(categoryDetail);
            httpRequest.setMethod(HttpConstant.Method.POST);
            map.put("newSign",newSignature);

            String params = JSON.toJSONString(map);
            logger.info("param is : [{}]",params);
            HttpRequestBody httpRequestBody = HttpRequestBody.custom(params.getBytes(),"application/json; charset=utf-8","utf-8");
            httpRequest.setRequestBody(httpRequestBody);

            httpRequest.addHeader("duuuid","fac3142579391d98");
            httpRequest.addHeader("duplatform","android");
            httpRequest.addHeader("appId","duapp");
            httpRequest.addHeader("duv","4.69.5");
            httpRequest.addHeader("duloginToken","");
            httpRequest.addHeader("timestamp",times);
            httpRequest.addHeader("Connection","Keep-Alive");
            httpRequest.addHeader("Host","app.dewu.com");
            httpRequest.addHeader("User-Agent","duapp/4.69.5(android;8.0.0)");
            httpRequest.addHeader("Accept-Encoding","gzip");

            HttpPage httpPage = downloader.download(httpRequest, httpConfig);
//            JSONObject jsonObject = JSONObject.parseObject(httpPage.getRawText());
//            logger.info(jsonObject.getString("msg"));
            logger.info(httpPage.getRawText());

        }

    }

    static class TestProductDetail implements Runnable{
        HttpClientDownloader downloader;
        HttpConfig httpConfig;
        String spuId;

        Set<String> spuIds = new HashSet<>();

        public TestProductDetail(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }
        public TestProductDetail(HttpClientDownloader downloader, HttpConfig httpConfig,String spuId) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
            this.spuId = spuId;
        }

        @Override
        public void run() {
            String times = String.valueOf(System.currentTimeMillis());
            logger.info("times is [{}]",times);
            Map<String,Object> map = new HashMap<>();
            map.put("spuId",spuId);
            map.put("groupFirstId",spuId);
            map.put("loginToken","");
//            map.put("abTests","[{\"name\":\"commodity_QA\",\"value\":\"0\"},{\"name\":\"evaluate_466_new\",\"value\":\"0\"}]");
            map.put("arFileSwitch",true);
            map.put("productSourceName","");
            map.put("propertyValueId","0");
            map.put("skuId","0");
            map.put("newSign","");
            map.put("platform","android");
            map.put("timestamp",times);
//            map.put("uuid","fac3142579391d97");
            map.put("uuid","fac3142579391d98");
            map.put("v","4.71.0");
            String newSignature = genNewSignature(map, scKey);
            logger.info("newSign is : [{}]",newSignature);

            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl(productDetail);
            httpRequest.setMethod(HttpConstant.Method.POST);
            map.put("newSign",newSignature);

            String params = JSON.toJSONString(map);
            logger.info("param is : [{}]",params);
            HttpRequestBody httpRequestBody = HttpRequestBody.custom(params.getBytes(),"application/json; charset=utf-8","utf-8");
            httpRequest.setRequestBody(httpRequestBody);

//            httpRequest.addHeader("duuuid","fac3142579391d97");
            httpRequest.addHeader("duuuid","fac3142579391d98");
            httpRequest.addHeader("duplatform","android");
            httpRequest.addHeader("appId","duapp");
//            httpRequest.addHeader("duv","4.69.5");
            httpRequest.addHeader("duv","4.71.0");
            httpRequest.addHeader("duloginToken","");
//            httpRequest.addHeader("shumeiid","20210426143326ac2c1e642fd9f8cefd4b868bba1d079701a3bfee661394af");
            httpRequest.addHeader("timestamp",times);
            httpRequest.addHeader("Connection","Keep-Alive");
            httpRequest.addHeader("Host","app.dewu.com");
            httpRequest.addHeader("User-Agent","duapp/4.71.0(android;8.0.0)");
            httpRequest.addHeader("Accept-Encoding","gzip");

            HttpPage httpPage = downloader.download(httpRequest, httpConfig);
            JSONObject jsonObject = JSONObject.parseObject(httpPage.getRawText());
            int status = jsonObject.getIntValue("status");
            logger.info(jsonObject.getString("msg"));
//            if (status != 200){
//                throw new Exception("downlaod failed");
//            }

            logger.info(httpPage.getRawText());

        }
    }

    static class TestWXProductDetail implements Runnable{
        HttpClientDownloader downloader;
        HttpConfig httpConfig;
        String spuId;

        public TestWXProductDetail(HttpClientDownloader downloader, HttpConfig httpConfig,String spuId) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
            this.spuId = spuId;
        }

        @Override
        public void run() {
            Map<String,Object> params = new HashMap<>();
            params.put("sign","41a45baf63208c58390783913699bc0e");
            params.put("spuId",spuId);
            params.put("productSourceName","");
            params.put("propertyValueId","0");
            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl(productDetailUrl);
            httpRequest.setMethod(HttpConstant.Method.POST);
            httpRequest.setRequestBody(HttpRequestBody.json(JSON.toJSONString(params),"utf-8"));
            httpRequest.addHeader("Host","app.dewu.com");
            httpRequest.addHeader("Connection","keep-alive");
            httpRequest.addHeader("User-Agent","Mozilla/5.0 (Linux; Android 8.0.0; MI 5s Build/OPR1.170623.032; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.62 XWEB/2797 MMWEBSDK/20210501 Mobile Safari/537.36 MMWEBID/1260 MicroMessenger/8.0.6.1900(0x28000635) Process/appbrand0 WeChat/arm64 Weixin NetType/WIFI Language/zh_CN ABI/arm64 MiniProgramEnv/android");
            httpRequest.addHeader("Content-Type","application/json");
            httpRequest.addHeader("platform","h5");
            httpRequest.addHeader("AppId","wxapp");
            httpRequest.addHeader("appVersion","4.4.0");
            httpRequest.addHeader("charset","utf-8");
            httpRequest.addHeader("Accept-Encoding","gzip,compress,br,deflate");
            httpRequest.addHeader("Referer","https://servicewechat.com/wx3c12cdd0ae8b1a7b/225/page-frame.html");
            httpRequest.addHeader("wxapp-login-token","");
            int i = 1;
            while (true){
                i++;
                try {
                    HttpPage httpPage = downloader.download(httpRequest, httpConfig);
                    logger.info(String.valueOf(httpPage.getStatusCode() + "-" + i));
                }catch (Exception e){
                    logger.error(e.getMessage(),e);
                    break;
                }

            }
        }
    }

    public static String genNewSignature(Map<String,Object> params, String scKey) {
        if(params == null){
            return null;
        }
        String sign = "";
        StringBuffer sb = new StringBuffer();
        try {
            // 1. 字典升序排序
            SortedMap<String,Object> sortedMap = new TreeMap<>(params);
            // 2. 拼按URL键值对
            Set<String> keySet = sortedMap.keySet();
            for(String key : keySet){
                //sign不参与算法
                if(key.equals("newSign")){
                    continue;
                }
                Object obj =  sortedMap.get(key);
                String value = obj.toString() ;
                if (obj instanceof Map){
                    value = JSON.toJSONString(obj);
                }
                if (obj instanceof List){
                    List list = (List) obj;
                    value = "";
                    for (Object o : list) {
                        if (value.length() > 0){
                            value += ",";
                        }
                        value += JSON.toJSONString(o);
                    }
                }
                if ("[]".equals(value)){
                    value = "";
                }
                sb.append(key).append(value);
            }
            String uriString = sb.toString();
            //AES运算
            AESUtils util = new AESUtils(scKey); // 密钥
            String encrypt = util.encryptData(uriString);
            encrypt = encrypt.replace("\r","").replace("\n","");//AES加密后存在换行符 需要去除后进行md5加密
            // 3. MD5运算得到请求签名
            sign = md5(encrypt);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return sign;
    }
}

